checkout: Support a "pure addition" mode

I plan to use this for `rpm-ostree livefs`.
https://github.com/projectatomic/rpm-ostree/issues/639

Closes: #714
Approved by: jlebon

diff --git a/man/ostree-checkout.xml b/man/ostree-checkout.xml
index 67d6469effa09f958cf3d4966fc08c29e94e7a07..c8585878daf0f160e29b8ab9844e912a651ddf51 100644 (file)
--- a/man/ostree-checkout.xml
+++ b/man/ostree-checkout.xml
@@ -89,6 +89,14 @@ Boston, MA 02111-1307, USA.
                 </para></listitem>
             </varlistentry>
 
+            <varlistentry>
+                <term><option>--union-add</option></term>
+
+                <listitem><para>
+                    Keep existing directories and files.
+                </para></listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--allow-noent</option></term>
 

diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c
index 50bc7030db57fee4b12b2fa37791a72a6a351928..53409529f3168b69ed0076fad92802760b5dfe5a 100644 (file)
--- a/src/libostree/ostree-repo-checkout.c
+++ b/src/libostree/ostree-repo-checkout.c
@@ -201,8 +201,16 @@ checkout_file_from_input_at (OstreeRepo     *self,
       while (G_UNLIKELY (res == -1 && errno == EINTR));
       if (res == -1)
         {
-          glnx_set_error_from_errno (error);
-          goto out;
+          if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)
+            {
+              ret = TRUE;
+              goto out;
+            }
+          else
+            {
+              glnx_set_error_from_errno (error);
+              goto out;
+            }
         }
 
       if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER)
@@ -240,6 +248,11 @@ checkout_file_from_input_at (OstreeRepo     *self,
       while (G_UNLIKELY (fd == -1 && errno == EINTR));
       if (fd == -1)
         {
+          if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)
+            {
+              ret = TRUE;
+              goto out;
+            }
           glnx_set_error_from_errno (error);
           goto out;
         }
@@ -332,6 +345,12 @@ checkout_file_unioning_from_input_at (OstreeRepo     *repo,
   return ret;
 }
 
+typedef enum {
+  HARDLINK_RESULT_NOT_SUPPORTED,
+  HARDLINK_RESULT_SKIP_EXISTED,
+  HARDLINK_RESULT_LINKED
+} HardlinkResult;
+
 static gboolean
 checkout_file_hardlink (OstreeRepo                          *self,
                         OstreeRepoCheckoutAtOptions           *options,
@@ -339,28 +358,32 @@ checkout_file_hardlink (OstreeRepo                          *self,
                         int                                  destination_dfd,
                         const char                          *destination_name,
                         gboolean                             allow_noent,
-                        gboolean                            *out_was_supported,
+                        HardlinkResult                      *out_result,
                         GCancellable                        *cancellable,
                         GError                             **error)
 {
-  gboolean ret = FALSE;
-  gboolean ret_was_supported = FALSE;
+  HardlinkResult ret_result = HARDLINK_RESULT_NOT_SUPPORTED;
   int srcfd = (self->mode == OSTREE_REPO_MODE_BARE || self->mode == OSTREE_REPO_MODE_BARE_USER) ?
     self->objects_dir_fd : self->uncompressed_objects_dir_fd;
 
  again:
-  if (linkat (srcfd, loose_path, destination_dfd, destination_name, 0) != -1)
-    ret_was_supported = TRUE;
+  if (linkat (srcfd, loose_path, destination_dfd, destination_name, 0) == 0)
+    ret_result = HARDLINK_RESULT_LINKED;
   else if (!options->no_copy_fallback && (errno == EMLINK || errno == EXDEV || errno == EPERM))
     {
       /* EMLINK, EXDEV and EPERM shouldn't be fatal; we just can't do the
        * optimization of hardlinking instead of copying.
        */
-      ret_was_supported = FALSE;
     }
   else if (allow_noent && errno == ENOENT)
     {
-      ret_was_supported = FALSE;
+    }
+  else if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)
+    {
+      /* In this mode, we keep existing content.  Distinguish this case though to
+       * avoid inserting into the devino cache.
+       */
+      ret_result = HARDLINK_RESULT_SKIP_EXISTED;
     }
   else if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES)
     { 
@@ -370,7 +393,7 @@ checkout_file_hardlink (OstreeRepo                          *self,
        * the same file, then rename() does nothing, and returns a
        * success status."
        *
-       * So we can't make this atomic.  
+       * So we can't make this atomic.
        */
       (void) unlinkat (destination_dfd, destination_name, 0);
       goto again;
@@ -379,14 +402,12 @@ checkout_file_hardlink (OstreeRepo                          *self,
     {
       g_prefix_error (error, "Hardlinking %s to %s: ", loose_path, destination_name);
       glnx_set_error_from_errno (error);
-      goto out;
+      return FALSE;
     }
 
-  ret = TRUE;
-  if (out_was_supported)
-    *out_was_supported = ret_was_supported;
- out:
-  return ret;
+  if (out_result)
+    *out_result = ret_result;
+  return TRUE;
 }
 
 static gboolean
@@ -439,7 +460,7 @@ checkout_one_file_at (OstreeRepo                        *repo,
     }
   else
     {
-      gboolean did_hardlink = FALSE;
+      HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED;
       /* Try to do a hardlink first, if it's a regular file.  This also
        * traverses all parent repos.
        */
@@ -469,11 +490,11 @@ checkout_one_file_at (OstreeRepo                        *repo,
                                            options,
                                            loose_path_buf,
                                            destination_dfd, destination_name,
-                                           TRUE, &did_hardlink,
+                                           TRUE, &hardlink_res,
                                            cancellable, error))
                 goto out;
 
-              if (did_hardlink && options->devino_to_csum_cache)
+              if (hardlink_res == HARDLINK_RESULT_LINKED && options->devino_to_csum_cache)
                 {
                   struct stat stbuf;
                   OstreeDevIno *key;
@@ -492,13 +513,13 @@ checkout_one_file_at (OstreeRepo                        *repo,
                   g_hash_table_add ((GHashTable*)options->devino_to_csum_cache, key);
                 }
 
-              if (did_hardlink)
+              if (hardlink_res != HARDLINK_RESULT_NOT_SUPPORTED)
                 break;
             }
           current_repo = current_repo->parent_repo;
         }
 
-      need_copy = !did_hardlink;
+      need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED);
     }
 
   can_cache = (options->enable_uncompressed_cache
@@ -514,7 +535,7 @@ checkout_one_file_at (OstreeRepo                        *repo,
       && repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2
       && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER)
     {
-      gboolean did_hardlink;
+      HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED;
       
       if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL,
                                   cancellable, error))
@@ -560,19 +581,20 @@ checkout_one_file_at (OstreeRepo                        *repo,
 
       if (!checkout_file_hardlink (repo, options, loose_path_buf,
                                    destination_dfd, destination_name,
-                                   FALSE, &did_hardlink,
+                                   FALSE, &hardlink_res,
                                    cancellable, error))
         {
           g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name);
           goto out;
         }
 
-      need_copy = !did_hardlink;
+      need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED);
     }
 
   /* Fall back to copy if we couldn't hardlink */
   if (need_copy)
     {
+      g_assert (!options->no_copy_fallback);
       if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs,
                                   cancellable, error))
         goto out;
@@ -655,7 +677,9 @@ checkout_tree_at (OstreeRepo                        *self,
   while (G_UNLIKELY (res == -1 && errno == EINTR));
   if (res == -1)
     {
-      if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES)
+      if (errno == EEXIST &&
+          (options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES
+           || options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES))
         did_exist = TRUE;
       else
         {

diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 648bd129092f895d0b0e3f662dd949b271e3a548..34685cc6c259d9a8d4e8d28073ab7040e7f3af08 100644 (file)
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -722,11 +722,13 @@ typedef enum {
 /**
  * OstreeRepoCheckoutOverwriteMode:
  * @OSTREE_REPO_CHECKOUT_OVERWRITE_NONE: No special options
- * @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, overwrite earlier files, but keep earlier directories
+ * @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, unlink() and replace existing files, but do not modify existing directories
+ * @OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES: Only add new files/directories
  */
 typedef enum {
   OSTREE_REPO_CHECKOUT_OVERWRITE_NONE = 0,
-  OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES = 1
+  OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES = 1,
+  OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES = 2, /* Since: 2017.3 */
 } OstreeRepoCheckoutOverwriteMode;
 
 _OSTREE_PUBLIC

diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c
index 95172f8b1c46bdf3837d0c2dd6f9e07df8aa777c..74e27cfb600b03e4e678abc1fa05ecf945f39610 100644 (file)
--- a/src/ostree/ot-builtin-checkout.c
+++ b/src/ostree/ot-builtin-checkout.c
@@ -36,6 +36,7 @@ static gboolean opt_allow_noent;
 static gboolean opt_disable_cache;
 static char *opt_subpath;
 static gboolean opt_union;
+static gboolean opt_union_add;
 static gboolean opt_whiteouts;
 static gboolean opt_from_stdin;
 static char *opt_from_file;
@@ -63,6 +64,7 @@ static GOptionEntry options[] = {
   { "disable-cache", 0, 0, G_OPTION_ARG_NONE, &opt_disable_cache, "Do not update or use the internal repository uncompressed object cache", NULL },
   { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" },
   { "union", 0, 0, G_OPTION_ARG_NONE, &opt_union, "Keep existing directories, overwrite existing files", NULL },
+  { "union-add", 0, 0, G_OPTION_ARG_NONE, &opt_union_add, "Keep existing files/directories, only add new", NULL },
   { "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL },
   { "allow-noent", 0, 0, G_OPTION_ARG_NONE, &opt_allow_noent, "Do nothing if specified path does not exist", NULL },
   { "from-stdin", 0, 0, G_OPTION_ARG_NONE, &opt_from_stdin, "Process many checkouts from standard input", NULL },
@@ -87,14 +89,23 @@ process_one_checkout (OstreeRepo           *repo,
    * `ostree_repo_checkout_at` until such time as we have a more
    * convenient infrastructure for testing C APIs with data.
    */
-  if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks)
+  if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks || opt_union_add)
     {
       OstreeRepoCheckoutAtOptions options = { 0, };
-      
+
       if (opt_user_mode)
         options.mode = OSTREE_REPO_CHECKOUT_MODE_USER;
-      if (opt_union)
+      /* Can't union these */
+      if (opt_union && opt_union_add)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Cannot specify both --union and --union-add");
+          goto out;
+        }
+      else if (opt_union)
         options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES;
+      else if (opt_union_add)
+        options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES;
       if (opt_whiteouts)
         options.process_whiteouts = TRUE;
       if (subpath)

diff --git a/tests/basic-test.sh b/tests/basic-test.sh
index 2698279631eb699cfe434208df72e00e33920f48..045e4217bf9fad325d0433943f1883625079c764 100644 (file)
--- a/tests/basic-test.sh
+++ b/tests/basic-test.sh
@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-echo "1..62"
+echo "1..63"
 
 $CMD_PREFIX ostree --version > version.yaml
 python -c 'import yaml; yaml.safe_load(open("version.yaml"))'
@@ -279,6 +279,24 @@ cd checkout-test2-union
 assert_file_has_content ./yet/another/tree/green "leaf"
 echo "ok checkout union 1"
 
+cd ${test_tmpdir}
+$OSTREE commit -b test-union-add --tree=ref=test2
+$OSTREE checkout test-union-add checkout-test-union-add
+echo 'file for union add testing' > checkout-test-union-add/union-add-test
+echo 'another file for union add testing' > checkout-test-union-add/union-add-test2
+$OSTREE commit -b test-union-add --tree=dir=checkout-test-union-add
+rm checkout-test-union-add -rf
+# Check out previous
+$OSTREE checkout test-union-add^ checkout-test-union-add
+assert_not_has_file checkout-test-union-add/union-add-test
+assert_not_has_file checkout-test-union-add/union-add-test2
+# Now create a file we don't want overwritten
+echo 'existing file for union add' > checkout-test-union-add/union-add-test
+$OSTREE checkout --union-add test-union-add checkout-test-union-add
+assert_file_has_content checkout-test-union-add/union-add-test 'existing file for union add'
+assert_file_has_content checkout-test-union-add/union-add-test2 'another file for union add testing'
+echo "ok checkout union add"
+
 cd ${test_tmpdir}
 rm -rf shadow-repo
 mkdir shadow-repo